{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "raw_weather_data = pd.read_csv(r\"weatherHistory.csv\", encoding = 'UTF-8',header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "after_drop=raw_weather_data.drop(columns=['Formatted Date','Precip Type','Apparent Temperature (C)','Loud Cover','Daily Summary'])\n",
    "idx = [i for i in range(0,96452,3)]\n",
    "after_drop = after_drop.loc[idx]\n",
    "def check_direction(degree: int) -> int:\n",
    "    deg = (degree + 22.5) % 360\n",
    "    return int(deg / 45)\n",
    "tmp = after_drop['Wind Bearing (degrees)'].apply(lambda x:check_direction(x))\n",
    "after_drop.drop(labels='Wind Bearing (degrees)', axis=\"columns\", inplace=True)\n",
    "after_drop['Wind Bearing (degrees)'] = tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "values = list(after_drop['Summary'].unique())\n",
    "value_index_dict = {value:idx for idx,value in enumerate(values)}\n",
    "after_drop['Summary'] = after_drop['Summary'].map(value_index_dict)\n",
    "after_drop = after_drop.tail(1682)\n",
    "after_drop = after_drop.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "      Summary  Temperature (C)  Humidity  Wind Speed (km/h)  Visibility (km)  \\\n0           0        33.861111      0.32            12.3004          16.1000   \n1           0        27.755556      0.53            11.2700          16.1000   \n2           5        25.255556      0.55            14.0392          16.1000   \n3           5        22.083333      0.65            14.2002          16.1000   \n4           0        22.133333      0.65            12.3004          10.2557   \n...       ...              ...       ...                ...              ...   \n1677        0        22.138889      0.65             7.7763          16.1000   \n1678        0        28.866667      0.37            13.9265          16.1000   \n1679        0        31.083333      0.28            15.5043          16.1000   \n1680        0        28.838889      0.32            12.2038          16.1000   \n1681        0        22.038889      0.56             8.9838          16.1000   \n\n      Pressure (millibars)  Wind Bearing (degrees)  \n0                  1009.87                       4  \n1                  1010.26                       3  \n2                  1010.09                       3  \n3                  1009.70                       4  \n4                  1009.61                       3  \n...                    ...                     ...  \n1677               1015.46                       1  \n1678               1015.35                       1  \n1679               1014.17                       1  \n1680               1014.07                       0  \n1681               1015.66                       1  \n\n[1682 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Summary</th>\n      <th>Temperature (C)</th>\n      <th>Humidity</th>\n      <th>Wind Speed (km/h)</th>\n      <th>Visibility (km)</th>\n      <th>Pressure (millibars)</th>\n      <th>Wind Bearing (degrees)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>33.861111</td>\n      <td>0.32</td>\n      <td>12.3004</td>\n      <td>16.1000</td>\n      <td>1009.87</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>27.755556</td>\n      <td>0.53</td>\n      <td>11.2700</td>\n      <td>16.1000</td>\n      <td>1010.26</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>25.255556</td>\n      <td>0.55</td>\n      <td>14.0392</td>\n      <td>16.1000</td>\n      <td>1010.09</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>5</td>\n      <td>22.083333</td>\n      <td>0.65</td>\n      <td>14.2002</td>\n      <td>16.1000</td>\n      <td>1009.70</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>22.133333</td>\n      <td>0.65</td>\n      <td>12.3004</td>\n      <td>10.2557</td>\n      <td>1009.61</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1677</th>\n      <td>0</td>\n      <td>22.138889</td>\n      <td>0.65</td>\n      <td>7.7763</td>\n      <td>16.1000</td>\n      <td>1015.46</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1678</th>\n      <td>0</td>\n      <td>28.866667</td>\n      <td>0.37</td>\n      <td>13.9265</td>\n      <td>16.1000</td>\n      <td>1015.35</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1679</th>\n      <td>0</td>\n      <td>31.083333</td>\n      <td>0.28</td>\n      <td>15.5043</td>\n      <td>16.1000</td>\n      <td>1014.17</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1680</th>\n      <td>0</td>\n      <td>28.838889</td>\n      <td>0.32</td>\n      <td>12.2038</td>\n      <td>16.1000</td>\n      <td>1014.07</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1681</th>\n      <td>0</td>\n      <td>22.038889</td>\n      <td>0.56</td>\n      <td>8.9838</td>\n      <td>16.1000</td>\n      <td>1015.66</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>1682 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "after_drop"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "after_drop.to_csv('./weather2.txt', sep='\\t', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip-code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>53</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>94043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>M</td>\n",
       "      <td>writer</td>\n",
       "      <td>32067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>43537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>15213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>26</td>\n",
       "      <td>F</td>\n",
       "      <td>student</td>\n",
       "      <td>33319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>32</td>\n",
       "      <td>M</td>\n",
       "      <td>administrator</td>\n",
       "      <td>02215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>20</td>\n",
       "      <td>M</td>\n",
       "      <td>student</td>\n",
       "      <td>97229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>48</td>\n",
       "      <td>F</td>\n",
       "      <td>librarian</td>\n",
       "      <td>78209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>22</td>\n",
       "      <td>M</td>\n",
       "      <td>student</td>\n",
       "      <td>77841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>943 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age gender     occupation zip-code\n",
       "0     24      M     technician    85711\n",
       "1     53      F          other    94043\n",
       "2     23      M         writer    32067\n",
       "3     24      M     technician    43537\n",
       "4     33      F          other    15213\n",
       "..   ...    ...            ...      ...\n",
       "938   26      F        student    33319\n",
       "939   32      M  administrator    02215\n",
       "940   20      M        student    97229\n",
       "941   48      F      librarian    78209\n",
       "942   22      M        student    77841\n",
       "\n",
       "[943 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_data = pd.read_csv(r\".\\raw_data\\u.user\", encoding = 'UTF-8',sep='|',header=None,names=['idx','age','gender','occupation','zip-code'])\n",
    "user_data.drop(labels='idx',axis=\"columns\", inplace=True)\n",
    "user_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip-code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.257576</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.696970</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>94043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.242424</td>\n",
       "      <td>M</td>\n",
       "      <td>writer</td>\n",
       "      <td>32067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.257576</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>43537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.393939</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>15213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>0.287879</td>\n",
       "      <td>F</td>\n",
       "      <td>student</td>\n",
       "      <td>33319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>0.378788</td>\n",
       "      <td>M</td>\n",
       "      <td>administrator</td>\n",
       "      <td>02215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>0.196970</td>\n",
       "      <td>M</td>\n",
       "      <td>student</td>\n",
       "      <td>97229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>0.621212</td>\n",
       "      <td>F</td>\n",
       "      <td>librarian</td>\n",
       "      <td>78209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>0.227273</td>\n",
       "      <td>M</td>\n",
       "      <td>student</td>\n",
       "      <td>77841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>943 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          age gender     occupation zip-code\n",
       "0    0.257576      M     technician    85711\n",
       "1    0.696970      F          other    94043\n",
       "2    0.242424      M         writer    32067\n",
       "3    0.257576      M     technician    43537\n",
       "4    0.393939      F          other    15213\n",
       "..        ...    ...            ...      ...\n",
       "938  0.287879      F        student    33319\n",
       "939  0.378788      M  administrator    02215\n",
       "940  0.196970      M        student    97229\n",
       "941  0.621212      F      librarian    78209\n",
       "942  0.227273      M        student    77841\n",
       "\n",
       "[943 rows x 4 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# user_data['age'] = (user_data['age'] - user_data['age'].min()) /(user_data['age'].max() - user_data['age'].min())\n",
    "# user_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in ['age','gender','occupation','zip-code']:\n",
    "    values = list(user_data[i].unique())\n",
    "    value_index_dict = {value:idx for idx,value in enumerate(values)}\n",
    "    user_data[i] = user_data[i].map(value_index_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip-code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>794</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>943 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  gender  occupation  zip-code\n",
       "0      0       0           0         0\n",
       "1      1       1           1         1\n",
       "2      2       0           2         2\n",
       "3      0       0           0         3\n",
       "4      3       1           1         4\n",
       "..   ...     ...         ...       ...\n",
       "938   17       1           5       791\n",
       "939   19       0           4        72\n",
       "940   23       0           5       792\n",
       "941   34       1          11       793\n",
       "942   27       0           5       794\n",
       "\n",
       "[943 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_data.to_csv('./data/user.txt', sep='\\t', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3 (ipykernel)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}